{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3111d0f2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-03T11:55:15.872569Z",
     "start_time": "2024-06-03T11:55:14.838336Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\miniconda3\\envs\\cnn\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import torch\n",
    "import torchvision\n",
    "import torch.nn as nn\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import DataLoader\n",
    "import torchvision.datasets as datasets\n",
    "import torchvision.transforms as transforms\n",
    "from torchvision.transforms.functional import InterpolationMode\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "06282f77",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-03T11:55:15.888110Z",
     "start_time": "2024-06-03T11:55:15.873569Z"
    }
   },
   "outputs": [],
   "source": [
    "class VGG16(nn.Module):\n",
    "    def __init__(self, num_classes=1000):\n",
    "        super(VGG16, self).__init__()\n",
    "        self.ConvBlock = nn.Sequential(\n",
    "            nn.Conv2d(in_channels=3, out_channels=64, kernel_size=3, stride=1, padding=1),\n",
    "            nn.BatchNorm2d(64),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(in_channels=64, out_channels=64, kernel_size=3, stride=1, padding=1),\n",
    "            nn.BatchNorm2d(64),\n",
    "            nn.ReLU(),\n",
    "\n",
    "            nn.MaxPool2d(kernel_size=2, stride=2),\n",
    "            nn.Conv2d(in_channels=64, out_channels=128, kernel_size=3, stride=1, padding=1),\n",
    "            nn.BatchNorm2d(128),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(in_channels=128, out_channels=128, kernel_size=3, stride=1, padding=1),\n",
    "            nn.BatchNorm2d(128),\n",
    "            nn.ReLU(),\n",
    "\n",
    "            nn.MaxPool2d(kernel_size=2, stride=2),\n",
    "            nn.Conv2d(in_channels=128, out_channels=256, kernel_size=3, stride=1, padding=1),\n",
    "            nn.BatchNorm2d(256),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(in_channels=256, out_channels=256, kernel_size=3, stride=1, padding=1),\n",
    "            nn.BatchNorm2d(256),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(in_channels=256, out_channels=256, kernel_size=3, stride=1, padding=1),\n",
    "            nn.BatchNorm2d(256),\n",
    "            nn.ReLU(),\n",
    "\n",
    "            nn.MaxPool2d(kernel_size=2, stride=2),\n",
    "            nn.Conv2d(in_channels=256, out_channels=512, kernel_size=3, stride=1, padding=1),\n",
    "            nn.BatchNorm2d(512),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(in_channels=512, out_channels=512, kernel_size=3, stride=1, padding=1),\n",
    "            nn.BatchNorm2d(512),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(in_channels=512, out_channels=512, kernel_size=3, stride=1, padding=1),\n",
    "            nn.BatchNorm2d(512),\n",
    "            nn.ReLU(),\n",
    "\n",
    "            nn.MaxPool2d(kernel_size=2, stride=2),\n",
    "            nn.Conv2d(in_channels=512, out_channels=512, kernel_size=3, stride=1, padding=1),\n",
    "            nn.BatchNorm2d(512),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(in_channels=512, out_channels=512, kernel_size=3, stride=1, padding=1),\n",
    "            nn.BatchNorm2d(512),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(in_channels=512, out_channels=512, kernel_size=3, stride=1, padding=1),\n",
    "            nn.BatchNorm2d(512),\n",
    "            nn.ReLU(),\n",
    "\n",
    "            nn.MaxPool2d(kernel_size=2, stride=2),\n",
    "#             nn.AvgPool2d(kernel_size=1, stride=1)\n",
    "        )\n",
    "        self.classifier = nn.Sequential(\n",
    "            nn.Linear(in_features=512 * 7 * 7, out_features=4096),\n",
    "            nn.ReLU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(in_features=4096, out_features=4096),\n",
    "            nn.ReLU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(in_features=4096, out_features=num_classes)\n",
    "        )\n",
    "        self.init_parameters()\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.ConvBlock(x)\n",
    "        x = x.view(-1, 512 * 7 * 7)\n",
    "        return self.classifier(x)\n",
    "\n",
    "    def init_parameters(self):\n",
    "        for layer in self.ConvBlock:\n",
    "            if isinstance(layer, nn.Linear) or isinstance(layer, nn.Conv2d):\n",
    "                nn.init.kaiming_normal_(layer.weight.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e9a4d4e2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-03T11:55:16.260579Z",
     "start_time": "2024-06-03T11:55:15.889320Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VGG16(\n",
       "  (ConvBlock): Sequential(\n",
       "    (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (2): ReLU()\n",
       "    (3): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (4): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (5): ReLU()\n",
       "    (6): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (7): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (8): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (9): ReLU()\n",
       "    (10): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (11): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (12): ReLU()\n",
       "    (13): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (14): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (15): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (16): ReLU()\n",
       "    (17): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (18): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (19): ReLU()\n",
       "    (20): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (21): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (22): ReLU()\n",
       "    (23): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (24): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (25): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (26): ReLU()\n",
       "    (27): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (28): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (29): ReLU()\n",
       "    (30): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (31): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (32): ReLU()\n",
       "    (33): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (34): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (35): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (36): ReLU()\n",
       "    (37): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (38): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (39): ReLU()\n",
       "    (40): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (41): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (42): ReLU()\n",
       "    (43): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "  )\n",
       "  (classifier): Sequential(\n",
       "    (0): Linear(in_features=25088, out_features=4096, bias=True)\n",
       "    (1): ReLU()\n",
       "    (2): Dropout(p=0.5, inplace=False)\n",
       "    (3): Linear(in_features=4096, out_features=4096, bias=True)\n",
       "    (4): ReLU()\n",
       "    (5): Dropout(p=0.5, inplace=False)\n",
       "    (6): Linear(in_features=4096, out_features=10, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = VGG16(num_classes=10)\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e1805139",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-03T11:55:16.276168Z",
     "start_time": "2024-06-03T11:55:16.261579Z"
    }
   },
   "outputs": [],
   "source": [
    "transform = transforms.Compose([\n",
    "    # transforms.RandomHorizontalFlip(),\n",
    "    transforms.Resize((224, 224), interpolation=InterpolationMode.BICUBIC),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "13348975",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-03T11:55:17.292388Z",
     "start_time": "2024-06-03T11:55:16.277168Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "train_dataset = datasets.CIFAR10(root='../data', train=True, transform=transform, download=True)\n",
    "test_dataset = datasets.CIFAR10(root='../data', train=False, transform=transform, download=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "fdfb3b02",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-03T11:55:17.308407Z",
     "start_time": "2024-06-03T11:55:17.293389Z"
    }
   },
   "outputs": [],
   "source": [
    "train_loader = DataLoader(dataset=train_dataset, batch_size=64, shuffle=True)\n",
    "test_loader = DataLoader(dataset=test_dataset, batch_size=64, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d38f5181",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-03T11:55:17.839404Z",
     "start_time": "2024-06-03T11:55:17.309917Z"
    }
   },
   "outputs": [],
   "source": [
    "model = VGG16(num_classes=10).cuda()\n",
    "criterion = nn.CrossEntropyLoss().cuda()\n",
    "optimizer = optim.SGD(model.parameters(), lr=1e-2, momentum=0.9, weight_decay=5e-4)\n",
    "scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.5, last_epoch=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "20ad2ef7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-03T11:55:17.855382Z",
     "start_time": "2024-06-03T11:55:17.840407Z"
    }
   },
   "outputs": [],
   "source": [
    "num_epochs = 20\n",
    "train_loss, train_step_loss, train_acc, test_loss, test_acc = [], [], [], [], []\n",
    "best_val_loss = float('inf')\n",
    "save_path = './VGG16.pth'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e5981f64",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-03T13:45:55.552431Z",
     "start_time": "2024-06-03T11:55:17.856384Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|                                                                                           | 0/20 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1|20, Train Loss:1.6840, Train Acc:38.24%\n",
      "Epoch 1|20, Test Loss:1.2366, Test Acc:56.29%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  5%|████                                                                            | 1/20 [05:33<1:45:33, 333.32s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2|20, Train Loss:1.1716, Train Acc:58.43%\n",
      "Epoch 2|20, Test Loss:0.9520, Test Acc:67.33%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 10%|████████                                                                        | 2/20 [11:05<1:39:49, 332.72s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3|20, Train Loss:0.9024, Train Acc:68.62%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 15%|████████████                                                                    | 3/20 [16:35<1:33:56, 331.56s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3|20, Test Loss:1.0131, Test Acc:66.94%\n",
      "Epoch 4|20, Train Loss:0.7424, Train Acc:74.39%\n",
      "Epoch 4|20, Test Loss:0.6966, Test Acc:76.55%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 20%|████████████████                                                                | 4/20 [22:10<1:28:42, 332.66s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 5|20, Train Loss:0.6216, Train Acc:78.67%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 25%|████████████████████                                                            | 5/20 [27:41<1:23:01, 332.11s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 5|20, Test Loss:0.7170, Test Acc:75.93%\n",
      "Epoch 6|20, Train Loss:0.5351, Train Acc:81.69%\n",
      "Epoch 6|20, Test Loss:0.6040, Test Acc:79.94%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 30%|████████████████████████                                                        | 6/20 [33:16<1:17:46, 333.31s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 7|20, Train Loss:0.4669, Train Acc:84.23%\n",
      "Epoch 7|20, Test Loss:0.5843, Test Acc:80.41%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 35%|████████████████████████████                                                    | 7/20 [38:50<1:12:14, 333.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 8|20, Train Loss:0.4084, Train Acc:86.36%\n",
      "Epoch 8|20, Test Loss:0.5482, Test Acc:81.74%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 40%|████████████████████████████████                                                | 8/20 [44:20<1:06:26, 332.19s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 9|20, Train Loss:0.3582, Train Acc:87.82%\n",
      "Epoch 9|20, Test Loss:0.5036, Test Acc:83.34%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 45%|████████████████████████████████████                                            | 9/20 [49:50<1:00:49, 331.75s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 10|20, Train Loss:0.3077, Train Acc:89.59%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 50%|████████████████████████████████████████▌                                        | 10/20 [55:20<55:09, 331.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 10|20, Test Loss:0.5124, Test Acc:83.34%\n",
      "Epoch 11|20, Train Loss:0.2750, Train Acc:90.58%\n",
      "Epoch 11|20, Test Loss:0.4880, Test Acc:85.12%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 55%|███████████████████████████████████████████▍                                   | 11/20 [1:00:49<49:34, 330.51s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 12|20, Train Loss:0.2425, Train Acc:91.69%\n",
      "Epoch 12|20, Test Loss:0.4806, Test Acc:85.04%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 60%|███████████████████████████████████████████████▍                               | 12/20 [1:06:18<44:01, 330.15s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 13|20, Train Loss:0.2180, Train Acc:92.51%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 65%|███████████████████████████████████████████████████▎                           | 13/20 [1:11:48<38:30, 330.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 13|20, Test Loss:0.5334, Test Acc:84.34%\n",
      "Epoch 14|20, Train Loss:0.1977, Train Acc:93.19%\n",
      "Epoch 14|20, Test Loss:0.4790, Test Acc:85.46%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 70%|███████████████████████████████████████████████████████▎                       | 14/20 [1:17:19<33:00, 330.16s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 15|20, Train Loss:0.1733, Train Acc:94.09%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 75%|███████████████████████████████████████████████████████████▎                   | 15/20 [1:22:49<27:31, 330.27s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 15|20, Test Loss:0.5504, Test Acc:83.70%\n",
      "Epoch 16|20, Train Loss:0.1514, Train Acc:94.73%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 80%|███████████████████████████████████████████████████████████████▏               | 16/20 [1:28:21<22:02, 330.68s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 16|20, Test Loss:0.5239, Test Acc:85.15%\n",
      "Epoch 17|20, Train Loss:0.1368, Train Acc:95.29%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 85%|███████████████████████████████████████████████████████████████████▏           | 17/20 [1:34:00<16:39, 333.23s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 17|20, Test Loss:0.5397, Test Acc:84.70%\n",
      "Epoch 18|20, Train Loss:0.1199, Train Acc:95.95%\n",
      "Epoch 18|20, Test Loss:0.4430, Test Acc:87.46%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 90%|███████████████████████████████████████████████████████████████████████        | 18/20 [1:39:35<11:07, 333.87s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 19|20, Train Loss:0.1122, Train Acc:96.21%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 95%|███████████████████████████████████████████████████████████████████████████    | 19/20 [1:45:06<05:32, 332.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 19|20, Test Loss:0.5308, Test Acc:85.85%\n",
      "Epoch 20|20, Train Loss:0.1029, Train Acc:96.47%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 20/20 [1:50:37<00:00, 331.88s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 20|20, Test Loss:0.5372, Test Acc:85.64%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "for epoch in tqdm(range(num_epochs)):\n",
    "    model.train()\n",
    "    temp_loss, temp_correct = 0, 0\n",
    "    for i, (images, labels) in enumerate(train_loader):\n",
    "        images, labels = images.cuda(), labels.cuda()\n",
    "        optimizer.zero_grad()\n",
    "        outputs = model(images)\n",
    "        loss = criterion(outputs, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "    \n",
    "        _, predicted = torch.max(outputs.data, 1)\n",
    "        temp_correct += (predicted == labels).sum().item()\n",
    "        temp_loss += loss\n",
    "        train_step_loss.append(loss.data.cpu().numpy())\n",
    "#         print('Epoch:{},Step:{},Train Loss:{:.4f}'.format(epoch+1, i+1, loss.item()))\n",
    "    print('Epoch {}|{}, Train Loss:{:.4f}, Train Acc:{:.2f}%'.format(epoch+1, num_epochs, temp_loss/len(train_loader), temp_correct/len(train_dataset) * 100))\n",
    "    train_loss.append(temp_loss/len(train_loader))\n",
    "    train_acc.append(temp_correct/len(train_dataset))\n",
    "    \n",
    "    temp_loss, temp_correct = 0, 0\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        for data in test_loader:\n",
    "            images, labels = data\n",
    "            images, labels = images.cuda(), labels.cuda()\n",
    "            outputs = model(images)\n",
    "            loss = criterion(outputs, labels)\n",
    "            _, predicted = torch.max(outputs.data, 1)\n",
    "            temp_correct += (predicted == labels).sum().item()\n",
    "            temp_loss += loss\n",
    "    print('Epoch {}|{}, Test Loss:{:.4f}, Test Acc:{:.2f}%'.format(epoch+1, num_epochs, temp_loss/len(test_loader), temp_correct/len(test_dataset) * 100))\n",
    "    test_loss.append(temp_loss/len(test_loader))\n",
    "    test_acc.append(temp_correct/len(test_dataset))\n",
    "    \n",
    "    if temp_loss < best_val_loss:\n",
    "        best_val_loss = temp_loss\n",
    "        torch.save(model.state_dict(), save_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "886bab3e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-03T13:45:56.053389Z",
     "start_time": "2024-06-03T13:45:55.554429Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1.6839888095855713, 1.171629786491394, 0.9024346470832825, 0.7423898577690125, 0.6215540766716003, 0.5351248979568481, 0.4668847918510437, 0.4083781838417053, 0.35821297764778137, 0.30774611234664917, 0.27501606941223145, 0.24253229796886444, 0.21802592277526855, 0.19771583378314972, 0.17325392365455627, 0.15140874683856964, 0.13677631318569183, 0.11988172680139542, 0.11220020800828934, 0.10290820896625519]\n",
      "[1.2366435527801514, 0.9519559144973755, 1.0130671262741089, 0.6966329216957092, 0.7170027494430542, 0.6040218472480774, 0.5843284726142883, 0.5481511354446411, 0.5035524964332581, 0.5123723149299622, 0.4880436658859253, 0.4805908203125, 0.5334358811378479, 0.4789722263813019, 0.5504263639450073, 0.523947536945343, 0.5397346019744873, 0.4430129826068878, 0.5308035612106323, 0.5371838212013245]\n"
     ]
    }
   ],
   "source": [
    "train_loss = [float(loss) for loss in train_loss]\n",
    "print(train_loss)\n",
    "test_loss = [float(loss) for loss in test_loss]\n",
    "print(test_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f0705a97",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-03T13:45:56.084935Z",
     "start_time": "2024-06-03T13:45:56.058389Z"
    }
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame({\n",
    "    'train_loss': train_loss,\n",
    "    'train_acc': train_acc,\n",
    "    'test_loss': test_loss,\n",
    "    'test_acc': test_acc\n",
    "})\n",
    "df.to_csv('experiment.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "577f2d1b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-03T13:52:58.497847Z",
     "start_time": "2024-06-03T13:52:58.347424Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAGwCAYAAABVdURTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABXqUlEQVR4nO3dd3wUZeLH8c+mJ5RQQgoQCESqVCkRPCsoooeCeAKiFAUbYuE4lZ8KYsPTExHlxEOKHUQFvcOGwYo0QRQVQ+8kFCEJCUlIdn5/PCQhkEp2M7ub7/v1mldmZ2dmn2EI++WZpzgsy7IQERER8RF+dhdARERExJUUbkRERMSnKNyIiIiIT1G4EREREZ+icCMiIiI+ReFGREREfIrCjYiIiPiUALsLUNWcTif79u2jVq1aOBwOu4sjIiIi5WBZFunp6TRs2BA/v9LrZqpduNm3bx+xsbF2F0NERETOwu7du2ncuHGp+1S7cFOrVi3A/OHUrl3b5tKIiIhIeaSlpREbG1vwPV6aahdu8h9F1a5dW+FGRETEy5SnSYkaFIuIiIhPUbgRERERn6JwIyIiIj6l2rW5Ka+8vDxOnDhhdzGkggIDA/H397e7GCIiYiOFm9NYlkVycjJHjx61uyhylurUqUN0dLTGMRIRqaYUbk6TH2wiIyMJCwvTF6QXsSyLzMxMDhw4AEBMTIzNJRIRETso3JwiLy+vINjUr1/f7uLIWQgNDQXgwIEDREZG6hGViEg1pAbFp8hvYxMWFmZzSaQy8u+f2kyJiFRPCjfF0KMo76b7JyJSvSnciIiIiE9RuBERERGfonAjZ4iLi2PatGl2F0NEROSsqLeUj7jkkkvo1KmTS0LJmjVrqFGjRuULJSIi1YZlQUYGHD0KTic0aWJfWRRuqgnLssjLyyMgoOxb3qBBgyookYiIeBLLgqwsE07ylyNHir4ua1tenjnXJZfAV19V+SUUULgpg2VBZqY9nx0WBuXp+DNixAi++eYbvvnmG1588UUA5s6dy8iRI/nkk0945JFH2LBhA1988QWxsbGMGzeOlStXkpGRQZs2bZgyZQq9e/cuOF9cXBz33Xcf9913H2B6H82aNYslS5bw+eef06hRI55//nmuueaaMsuWl5fHbbfdxrJly0hOTqZJkybcdddd3HvvvUX2mzNnDs8//zxbtmyhXr16DBw4kJdffhmAo0eP8uCDD7J48WJSU1M555xzeOaZZ/jrX/9azj9JEZHqITv7zOBRVjA5dXtOTuXLUI7/Q7udBxTBs2VmQs2a9nz2sWNQnqdDL774Ips2baJdu3Y8/vjjAPz2228APPTQQ/zrX/+iefPm1K1bl927d3PVVVfx1FNPERwczBtvvEG/fv1ISkqiSSl1iJMnT+bZZ5/lueee46WXXmLo0KHs3LmTevXqlVo2p9NJ48aNWbhwIfXr1+eHH37gtttuIyYmhhtuuAGAV155hXHjxvHMM8/Qt29fUlNTWb58ecHxffv2JT09nbfeeov4+Hh+//13Dc4nIj7r+HHYvbt8oeT0JSur8p/v5wd16hQudesWfV3a9rp1ITS0fP8xdyeFGx8QHh5OUFAQYWFhREdHA/DHH38A8Pjjj3P55ZcX7FuvXj06duxY8PqJJ55g0aJFfPzxx9x9990lfsaIESMYMmQIAE8//TTTp09n9erVXHnllaWWLTAwkMmTJxe8btasGStWrOC9994rCDdPPvkkf//734vU5nTr1g2AL7/8ktWrV7Nx40ZatmwJQPPmzcv+QxER8XBHj8LGjWcu27ebpwZny+GA8PDiA0l5wknNmvaHk8pSuClDWJipQbHrsyura9euRV4fO3aMxx57jCVLlrB//35yc3M5fvw4u3btKvU8HTp0KFivUaMGtWvXLpjDqSwzZsxgzpw57Nq1i+PHj5OTk0OnTp0AM03Cvn376NWrV7HHrl+/nsaNGxcEGxERb2JZsH9/8SEmObnk42rWhHr1Sq41KS2s1Kplal+qM4WbMjgc5Xs05KlO7/U0fvx4li5dyr/+9S/OOeccQkNDuf7668kp40FrYGBgkdcOhwOn01nm58+fP5/x48fz/PPP06NHD2rVqsVzzz3HqlWrgMK5oEpS1vsiIp4gLw927Cg+xKSmlnxco0bQps2ZS2Sk99ee2EnhxkcEBQWRl99MvRTLly9nxIgRDBgwADA1OTt27HBbuZYvX07Pnj256667CrZt3bq1YL1WrVrExcWRmJjIpZdeesbxHTp0YM+ePWzatEm1NyJiu9RU2LoVtmwpGmA2bSq5vYufHzRvfmaAad3aPD4S11O48RFxcXGsWrWKHTt2ULNmzRJrVVq0aMGHH35Iv379cDgcPProo+WqgTlbLVq04I033uDzzz+nWbNmvPnmm6xZs4ZmzZoV7PPYY49xxx13EBkZWdB4ePny5YwdO5aLL76Yiy66iIEDBzJ16lTOOecc/vjjDxwOR5ntfUREKsrpNI+Rtm4tumzbZn4ePlzyscHB0KrVmSGmRQsICam6axCFG58xfvx4hg8fTtu2bTl+/Dhz584tdr+pU6dyyy230LNnTyIiInjwwQdJS0tzW7luv/12fvrpJwYNGoTD4WDIkCHcddddfPrppwX7DB8+nKysLF544QXGjx9PREQE119/fcH7H3zwAePHj2fIkCFkZGQUdAUXETkb2dnmEVJx4WXbtrJ7HEVGQny8CTJt2xaGmLg4UEdOz+CwrMq0yfY+aWlphIeHk5qaSu3atYu8l5WVxfbt22nWrBkhitleS/dRRFJTzaOj/NBy6rJ7d+m9kfz9oWlTE2Di480jpVPXa9WquuuQQqV9f59ONTciIuKVLAv27DFtXv74o+iyf3/px9aoURhYTg8wTZrAaX0oxMso3Eil3HHHHbz11lvFvnfTTTcxc+bMKi6RiPiarKzCBrynBpikJDOXUUmiokoOMOqN5NsUbqRSHn/8ccaPH1/se2VVG4qInOrwYRNaTg8x27ebhr7FCQgwDXZbtzZLfi+kVq1A/wRVXwo3UimRkZFERkbaXQwR8SKpqbB6NfzyS9EQc+hQyceEhxcGl1ODTLNmeoQkZ1K4ERERt8nLg99+g5UrzbJqlamZKalBb5MmRWtg8peoKD1GkvJTuBEREZdJTi4MMStXwpo1xbeLadYMunQpDDFt2kDLlt49Irx4DoUbERE5K1lZ8NNPRWtldu48c79ataBbNzj/fLMkJJgGvSLuonAjIiJlsiwzZsypQWb9ejhxouh+Dgece27RINOmjQa3k6qlcCMiImc4eNCEl/zHS6tWFd/gNzKyMMScfz507apeSmI/hRsRkWrs0CHT4PfU5fffTbg5XVAQnHdeYZA5/3wzkq8a+oqnUbjxEZdccgmdOnVi2rRpLjnfiBEjOHr0KIsXL3bJ+UTEXocPnxlifvut+BADJrA0bw7duxfWzHTqZCaHFPF0CjciIj7kzz+LDzEHDpR8TLNmZgLIc88tXFq3Vs8l8WJWNZOammoBVmpq6hnvHT9+3Pr999+t48ePF250Oi3rxDF7FqezXNc0fPhwCyiybN++3dqwYYN15ZVXWjVq1LAiIyOtm266yTp48GDBcQsXLrTatWtnhYSEWPXq1bN69eplHTt2zJo0adIZ5/vqq6/KLMcDDzxgtWjRwgoNDbWaNWtmPfLII1ZOTk6RfT7++GOra9euVnBwsFW/fn2rf//+Be9lZWVZDzzwgNW4cWMrKCjIio+Pt1577bVy/Rmcqtj7KOJj0tMt6/vvLWvmTMsaO9ayLrvMsqKiLMs0/S1+iYuzrKuusqx//MOy5s2zrDVrLOvYMbuvRKR8Svv+Pp1qbsqSlwnv1bTns284BgFl/9fpxRdfZNOmTbRr147HH38cgMDAQLp3786oUaN44YUXOH78OA8++CA33HADy5YtY//+/QwZMoRnn32WAQMGkJ6eznfffYdlWYwfP56NGzeSlpbG3LlzAahXr16Z5ahVqxbz5s2jYcOGbNiwgdGjR1OrVi0eeOABAJYsWcKAAQN4+OGHeeONN8jJyeGTTz4pOH7YsGGsWLGC6dOn07FjR7Zv386h0oYsFakm0tNNl+u1awuXpKSSB8Jr2vTMmpg2baCmTf+UiVQ1hRsfEB4eTlBQEGFhYURHRwPw5JNP0rlzZ55++umC/ebMmUNsbCybNm3i2LFj5Obmct1119G0aVMA2rdvX7BvaGgo2dnZBecrj0ceeaRgPS4ujvHjxzN//vyCcPPUU08xePBgJk+eXLBfx44dAdi0aRPvvfceS5cupXfv3gA0b968on8UIl4vLe3MILNpU/FBpmFD6NDhzBBTq1bVl1vEkyjclMU/zNSg2PXZZ+nnn3/mq6++omYx/1XbunUrV1xxBb169aJ9+/b06dOHK664guuvv566deue9WcuWLCA6dOns3Xr1oLwdOrkmevXr2f06NHFHrt+/Xr8/f25+OKLz/rzRbzN6UHmxx9h8+big0zjxmZE31OXqKiqL7OIN1C4KYvDUa5HQ57m2LFj9OvXj3/+859nvBcTE4O/vz9Lly7lhx9+4IsvvuCll17i4YcfZtWqVTRr1qzCn7dixQqGDh3K5MmT6dOnD+Hh4cyfP5/nn3++YJ/Q0NASjy/tPRFfkJpafI1McWJji4aY885TkBGpCIUbHxEUFEReXl7B6/POO48PPviAuLg4AgKKv80Oh4MLLriACy64gIkTJ9K0aVMWLVrEuHHjzjhfWX744QeaNm3Kww8/XLBt52njsHfo0IHExERGjhx5xvHt27fH6XTyzTffFDyWEvFWlgW//gqJiWbwu7VrTY1McU4PMl26aGoCkcpSuPERcXFxrFq1ih07dlCzZk3GjBnDrFmzGDJkCA888AD16tVjy5YtzJ8/n9dee40ff/yRxMRErrjiCiIjI1m1ahUHDx6kTZs2Bef7/PPPSUpKon79+oSHhxMYGFji57do0YJdu3Yxf/58unXrxpIlS1i0aFGRfSZNmkSvXr2Ij49n8ODB5Obm8sknn/Dggw8SFxfH8OHDueWWWwoaFO/cuZMDBw5www03uPXPTsQV9u+HL7+EpUvNkpx85j5NmpwZZBo0qPqyivg893fe8iwV7gruJZKSkqzzzz/fCg0NLegKvmnTJmvAgAFWnTp1rNDQUKt169bWfffdZzmdTuv333+3+vTpYzVo0MAKDg62WrZsab300ksF5ztw4IB1+eWXWzVr1ix3V/B//OMfVv369a2aNWtagwYNsl544QUrPDy8yD4ffPCB1alTJysoKMiKiIiwrrvuuoL3jh8/bt1///1WTEyMFRQUZJ1zzjnWnDlzKvxn4c33UbzHsWOW9cknlnX//ZbVrt2Z3a5DQy2rTx/LevJJy/rsM8s6cMDuEot4t4p0BXdYVkmdCX1TWloa4eHhpKamFmnsCpCVlcX27dtp1qwZISEhNpVQKkv3UdwhLw/WrSusmfnhB8jJKXzf4TBtYy6/3Cw9e4L++om4Tmnf36fTYykRkRLs2FEYZhITzei/p2rSpDDM9OoFERG2FFNETqNwI+Xy9NNPFxkz51QXXnghn376aRWXSMT1jh6Fr74qDDRbthR9v3ZtuPTSwkDTooUmjRTxRAo3Ui533HFHiQ171Y1bvNWJE6Y309Kl8MUXsHo1OJ2F7/v7m0kj88NM9+5QQudDEfEg+jWVcqlXr165pmAQ8WQZGbByJXz3nVlWroTMzKL7tGxZGGYuuQTCw20pqohUgp/dBZgxYwZxcXGEhISQkJDA6tWrS9z3xIkTPP7448THxxMSEkLHjh357LPPXF4m56n/dROvo/sn+Q4fho8+gvHjISEB6tSB3r1h8mRYtswEm/r1YdAgeO012LnTzNn08stw7bUKNiLeytaamwULFjBu3DhmzpxJQkIC06ZNo0+fPiQlJRFZzChWjzzyCG+99RazZs2idevWfP755wwYMIAffviBzp07V7o8QUFB+Pn5sW/fPho0aEBQUBAOPVD3GpZlkZOTw8GDB/Hz8yMoKMjuIkkV2727sFbmu+/gt9/O3Cc2Fi68sHBp0wb8bP9vnoi4kq1dwRMSEujWrRsvv/wyYP7HHRsby9ixY3nooYfO2L9hw4Y8/PDDjBkzpmDbwIEDCQ0N5a233ir2M7Kzs8nOzi54nZaWRmxsbIldyXJycti/fz+Zp9dVi9cICwsjJiZG4cbHWZapZfnuO/j2W/PztEGxAWjdujDIXHSRmTFbRLyPV3QFz8nJYe3atUyYMKFgm5+fH71792bFihXFHpOdnX3GuCWhoaF8//33JX7OlClTisxCXZagoCCaNGlCbm5uhaYfEM/g7+9PQECAatx8UG4urF9fWCvz/fdw8GDRffz9oXPnwjDzl79oBGCR6si2cHPo0CHy8vKIOm02uKioKP74449ij+nTpw9Tp07loosuIj4+nsTERD788MNSQ8iECRMYN25cwev8mpvSOBwOAgMDS51uQETcb8sWmD/fhJkffoBjx4q+HxJi2tLk18qcfz7UqmVPWUXEc3hVb6kXX3yR0aNH07p1axwOB/Hx8YwcOZI5c+aUeExwcDDBwcFVWEoRqQzLMl2zp0+HTz4xr/OFh5vamPyamS5dQL/eInI628JNREQE/v7+pKSkFNmekpJCdHR0scc0aNCAxYsXk5WVxeHDh2nYsCEPPfQQzZs3r4oii4gbZWTAm2+aULNxY+H2vn3hqqtMzUy7dmr8KyJls+2fiaCgILp06UJiYmLBNqfTSWJiIj169Cj12JCQEBo1akRubi4ffPAB1157rbuLKyJusmMH/OMf0Lgx3HmnCTa1asE998CmTab25u67oUMHBRsRKR9bH0uNGzeO4cOH07VrV7p37860adPIyMhg5MiRAAwbNoxGjRoxZcoUAFatWsXevXvp1KkTe/fu5bHHHsPpdPLAAw/YeRkiUkGWBd98Y2ppPvqocFTg+HgYOxZGjjRTHYiInA1bw82gQYM4ePAgEydOJDk5mU6dOvHZZ58VNDLetWsXfqf8Vy0rK4tHHnmEbdu2UbNmTa666irefPNN6tSpY9MViEhFHD8O77xjQs0vvxRuv/xyU1Nz1VWqnRGRyrN1nBs7VKSfvIi4xp498O9/w3/+Y0YNBggLg2HDTE1N27b2lk9EPJ9XjHMjIr7NsmDFClNL8/77kD9iQ9Ompg3NrbdC3br2llFEfJPCjYi4VHY2vPeeCTU//li4/eKL4d57oV8/zawtIu6lf2JExCWSk2HmTLPkj/AQHAxDh5r2NB072ls+Eak+FG5E5KxZlqmdmT4dFiyAEyfM9kaN4K674LbbICLC3jKKSPWjcCMiFbZ9u+n19M478Pvvhdt79jS1NNddB5q9RETsonAjIuVy8KBpS/POO2aep3zBwfC3v5lQ062bfeUT8RmWE06kQ1C43SXxWgo3IlKiY8fMIHtvvw1ffFHY48nPDy67DG680dTShOvfYBHXOLoBVoyAIz9BTB+IHwWN+oF/kN0l8yoKNyJSxIkTJsi8/bYJNpmZhe917WoCzeDBEBNjXxlFfI4zFzY+CxseA+fJxmv7PzNLSCQ0Gw7xt0LtVrYW01so3IgITqcZk+btt82jp/yB9sBMiTB0qAk1rfTvqojrpf4OK4bDnyfHTmjUD879P9jzEWybB1nJsPE5szS40NTmNLkeAsJsLbYn0wjFItXYb7+ZQPPOO7BzZ+H2yEhTOzN0qGlH43DYV0YRn+XMhT+eh18mgjMHAutA1+kQd1PhL53zBOz7BLa8Bvs/Me1xAALDIW6oCTr1Ott2CVWpIt/fCjci1czu3fDuuybUnDq/U82apv3M0KGmPY0G2hNxo9Q/YOUIOLzKvG54FXSfBWENSz4mc6+pydk6GzK2F26vex6cMxqaDvHpRsgKN6VQuJHq6M8/zRQIb78N335buD0wEPr2NYGmXz8IDbWvjCLVgjMPkqbBzw+DMxsCa0OXF02bmvJWkVpOSFlmanP2LDK1PgD+odDkBlOb0+ACn6tyVbgphcKNVCcbNsATT8DixYUD7AFcdJEJNNdfD/Xq2VY8keolbROsHAmHTo6lENMHEl6DsMZnf86sQ7DjLdg6y7TdyVe7tQk5zYZBSIPKldtDKNyUQuFGqoPNm2HSJJg/34wiDNChgwk0Q4ZAbKy95ROpViwnJE2HnydAXhYE1ILzppreT66qXbEsOLQStr4GO+dD3slujn6B0OhaE3RiLgeHn2s+zwYKN6VQuBFftnMnPP44vP564Zg0f/sbPPyw5nYSsUX6Flh5Cxz8zryO7g0Js6FGE/d95ok02LkAtsyCP9cUbq/RFJrfAs1HQg3v+x+Owk0pFG7EF+3fD089Bf/5T+Hjp7/+1TyS6tTJ1qKJVE+WEzbNgPUPQt5xCKgJnf8F59xWtW1hjvxsGiBvfxNOHDXbHH4Q3t40Pg6oBYG1zvxZ3LZT3/MPq/I2PQo3pVC4EV9y6BD885/w8suQlWW29eplQk2PHvaWTaTaOrbN1NYc+Ma8jroUEuZAzTj7ypR73DQ+3jILDnxd+fM5/ExgC6xdfAgKbwPnTqj855yiIt/f6uwp4oVSU2HqVHjhBUhPN9t69DC1N5deam/ZxAdk7IT9SyFzDzQZCHXa210i72A5Ycur8NM/IDfD1G50fg5a3GF/W5eAUIi70SzHtkNaEuSmmzmsTqSbR1n5r0v6eSIdco8B1sn5r9LMUpyIni4PNxWhcCPiRTIy4KWX4Nln4cgRs61zZ3jySdOl28d6fkpVOZEOKV9D8hew/wtI31T43q+TIfISaHWPGTnXT18bxcrYCStvhZRE8zryIlNbUyve3nIVp2Yzs5wNywm5mcWHoVPXQ6JcW+YK0t9SES+QlQWvvgpPPw0HDphtbdqYx08DBpiJLKWCTqTBttfh6M/gCDC9ShyB5uepy+nbKvo6NAaC6tp9tUU588xQ/8lLTZg5tAKs3ML3Hf5QPwGCI2DfEvMY48DXENYEWo4xPW+CPXQMgew/YfcHsHsR4DTdrEMbm5+nLoHhrvnfgGWZbtjr/m5qNfxDodMz0PJu+2tr3MHhB4E1zeLBFG5EPNiJEzB3rgkxe/aYbc2bw+TJpku3v7+95fNKaZth00uwbe7JKvYqENoQws+F8HZQ5+TP8LamfUJVObajMMykJELOkaLv1zzHdBWOvsK0Eckf6TZjN2yZaR63ZO4yDWQ3PGaG/m85Fup2qLprKEluJuz9L+x4B/Z/WjjxZGkCapQcfPK3B9cvPQBl7IZVt5o/VzAD5yXMhdotXHNdctbUoFjEA+XlmfmeHnsMtm0z2xo3hokTYcQIM7KwVIBlmS+gpOlmnh5O/rNXuw00+Zv536jzROFi5Z78eeK07RV87cyBnD9LLleNpqeEnnZmvXYb0z6isk6kQcpXpu1M8heQvrno+4HhEN3LhJmYy6Fm89LPl5dlxk9Jmg5HfircHnnxyUdW11TtIytnLiR/aQLNnkVFg2qdjhA3xDwaydxTdDm+B7IPl3zeU/kFFx96whrD8X1m3JoTaeAfAh2fhpb3gJ/+x+Eu6i1VCoUb8WROJyxaZELM7ycHG42MNOPU3HYbhITYWz6vk5thusAmTYe0jYXbG14Fre6F6Mvd31DpRJoZOTb1Nzj6K6T+ataP7y/hAAfUjC8MO/nBp1ZL8A8q+XOcueZR0/4vTJA7tAKsvFNO6w8RPcw1x1wB9bqeXRixLDPCbtJ08/gn/zPCYk95ZFW/4uct92evhJ3vmHFcsg8WvlcjzjSWbXqjqR0rTe5xOL63+OCTv56VUr4y1T8fesyD2q3O9qqknBRuSqFwI57IsuDTT+GRR+Cnk/8prlsXHngAxo6FGjXsLZ/XydhpxhjZMqtwbI+AmmbwspZjPeOxQfafJuSk/gpHT/5M/bXkWgVHANRuWRh4ws81XYsPn2w7k5xYeK35arUoDDORl7h+UsXMPbD55COr7ENmm3/IKY+sXDRyZOrvpoZmxztFJ4wMbgBNB5lAE3G+a4NqXo6pnSku+GTuMaE1/hZodb9qa6qIwk0pFG7E03z/PTz4IPxwcrqZmjVh3DizhPvuBL+uZ1lmFNikF2HPYtOrA0wtSMux0HyE58+YbFmQdaCwdufUmp6SutyeKrCOGQE35nITas62R0xF5WWZmpSk6XBkXeH2yIvMn33j/hWvJcrYbR6D7XjbNPrOF1ATGg8wtTTRvUyjbakWFG5KoXAjnuLQIVMzM3eueR0aCnffbbZFRNhbNq9S0BbkRTiyvnB7VC/z6KnhVd7/P2vLMrUF+TU9+cHn2FZTgxN9xSmPmmy8Vssyj8OSpsPu94s+smpxJ8SPhpBS/nJn/wm7FprHTgdOmb7eLxBi+ppA06gfBIS59zrEIynclELhRuzmdMK8efCPf8CfJ9uajhpl5oSKibG1aN4lcx9sfuXkI5GTbS/8Q6HZzaa2oE47e8tX3WXuPeWR1cn74xdsHlm1Ggt1O5ltuZmw52MTaPZ/VrSnU+TFJtDEXu+5Xc+lyijclELhRuz0229w553w3ck59Nq3N+PXaKqECji02tTS7HqvcGyWqmjMKmcnLwt2vgebpsOfawu3N7jQTB65Z7Fp+J2vbicTgJoM8srJHcV9FG5KoXAjdsjMNGPV/OtfkJtrGghPngz33KNu3eXiPAG7PjCh5vDKwu0N/mIePZ1Nmw6pWvk9nTZNh13vFx00sGZz0yg4bogZ/0ekGJpbSsSDLFli2tLs2GFe9+8PL74ITZrYWSoXyT0OKcvMcOvukr7ZDCJ3fJ957RcETQebsVXqdXHf54prORzQoIdZOj9vZqrOTYfY68xoyJo7RFxI4UbETfbsgXvvhQ8/NK+bNDHzQl1zjb3lcokj682X0/a3zux+7C4h0aZR6jm3Q6i989ZIJYU1hPaP2l0K8WEKNyIulptrQszEiXDsmJkiYdw4mDTJy8eryUk1jT63zi7adiKsCdQ6x32f6x9mxjJpckPpg9iJiJykcCPiQqtWwe23w88nh+Xo2RNmzjQNh71S/tgxW2ebLrp5x812v0Az1kj8KDPWiC9OECgiXkvhRsQFjhyB//s/0/PJsszows8+C7fc4qUzdh9Phu2vw9Y5kL6pcHv4uSbQxN1U+nglIiI2UrgRqQTLMhNcjhsHBw6YbcOHw3PPQYMG9patwpy5ZpyRrbPNDMv5A7AF1DQNeONHQf3uavgpIh5P4UbkLCUlwV13wbJl5nXr1vDKK3DJJbYWq+LSt8K2ObBtXmGPJDCTLMaPMm1dAmvaVjwRkYpSuBGpoKwsmDIFnnkGcnLMTN2PPgrjx0OQt7R3zcuC3R+aWpqUZYXbgyOg2TCIv1XjjYiI11K4EamApUtNbc2WLeb1lVfCjBnQvLm95Sq3Iz/D1tfMZIQ5R05udJh5ieJHQaNr1CNJRLyewo1IOSQnm3Y1775rXjdsaAbiGzjQC5qg5KTCzndPduH+sXB7WBOIvwWajzTD4IuI+AiFG5EyvPEGjB0LaWmm59Pdd5upFDx69o7sP2Hvx2bKguQvwJljtvsFmqkK4keZWbO9fbZsEZFiKNyIlCA9HcaMgTffNK+7djVj1nTx1BH/jyebSQh3fwApXxX2doKTXbhvhbib1YVbRHyewo1IMdatg8GDYfNmU1szeTJMmGBGG/YoGbtMw+DdH8LB74FT5sGt0xFiB5q5e8LbesHzMxER11C4ETmFZZmpE/7xD9MTqnFj087mL3+xu2SnSNsMez40j5z+XFP0vfrdCwONO6dEEBHxYAo3IicdPgwjR8J//2teX3stzJkD9erZWy4sC1J/M4+bdn8ARzec8qYDIi80gabxAKgRa1sxRUQ8hcKNCPDtt3DjjbB3rxmrZupU0+Xbtic5lmUmp9z9gXnkdOoUCI4AiLrM1M407q8ZskVETqNwI9VaXh489ZRpU+N0QsuWsGABdOpkQ2EsJxz8wYSZPR9Cxs7C9/yCzVg0sQOhUT8Itrs6SUTEc9k+pd+MGTOIi4sjJCSEhIQEVq9eXer+06ZNo1WrVoSGhhIbG8v9999PVlZWFZVWfMnevdCrF0yaZILN8OGwdm0VBZu8bDPtQcrXsP1NWHMXLGoEX14ISS+YYBNQA5r8DS6YDwMPwsUfQ/PhCjYiImWwteZmwYIFjBs3jpkzZ5KQkMC0adPo06cPSUlJREZGnrH/O++8w0MPPcScOXPo2bMnmzZtYsSIETgcDqZOnWrDFYi3WrLEhJnDh6FGDdPF+6abXHRyZx5k7YeM3ZC5GzJ3nbJ+8nXWgeKPDQw3owTHXgcxfSAg1EWFEhGpPhyWZVll7+YeCQkJdOvWjZdffhkAp9NJbGwsY8eO5aGHHjpj/7vvvpuNGzeSmJhYsO3vf/87q1at4vvvvy/XZ6alpREeHk5qaiq1PXoUNnGH7GzTpfuFF8zrzp1h/nzzOKpcLAuyDxUGlYyTYeXU18f3Fh1jpiT+oWZk4LBYqNXChJqoyzT9gYhIMSry/W1bzU1OTg5r165lwoQJBdv8/Pzo3bs3K1asKPaYnj178tZbb7F69Wq6d+/Otm3b+OSTT7j55ptL/Jzs7Gyys7MLXqelpbnuIsSrbNlixq5Zu9a8vu8+M/llcHAZBx5aBRsmw7EtJsDkleMxqCMAwhqZ4BLWxPRiOn09qJ7GnhERcQPbws2hQ4fIy8sjKqpoT4+oqCj++OOPYo+58cYbOXToEH/5y1+wLIvc3FzuuOMO/u///q/Ez5kyZQqTJ092adnF+7zzDtx+Oxw7Zrp2z5sH/fqV48CUr+Gbv0JuRtHtIdEmoNQ4GVgK1k++DonS1AYiIjbxqt5SX3/9NU8//TT//ve/SUhIYMuWLdx777088cQTPProo8UeM2HCBMaNG1fwOi0tjdhYjQVSXWRkmHmh5s41ry+6CN5+2wzOV6bkL+GbayDvOERfDuc+bAJMaCPwL6u6R0RE7GJbuImIiMDf35+UlJQi21NSUoiOji72mEcffZSbb76ZUaNGAdC+fXsyMjK47bbbePjhh/HzO7PzV3BwMMFlPncQX/TzzzBoECQlmSkUJk6ERx4p5xQK+z6FbweAMxsaXg0Xvg/+IW4vs4iIVJ5tXcGDgoLo0qVLkcbBTqeTxMREevToUewxmZmZZwQY/5PfVDa2ixYPY1nw739DQoIJNg0bwrJlpst3uYLNno/h2/4m2DTuDxd+qGAjIuJFbH0sNW7cOIYPH07Xrl3p3r0706ZNIyMjg5EjRwIwbNgwGjVqxJQpUwDo168fU6dOpXPnzgWPpR599FH69etXEHKkejtyBG69FRYtMq//+lfzSCqivBNh7/oAlg8GK9eMMdPzbfALdFt5RUTE9WwNN4MGDeLgwYNMnDiR5ORkOnXqxGeffVbQyHjXrl1FamoeeeQRHA4HjzzyCHv37qVBgwb069ePp556yq5LEA+yfLmZQmHXLggMhOeeg3vuqUCHpB3vwoqbTTfupjdCj9fBz6uapYmICDaPc2MHjXPje5xO06V74kQzncI555ixa7p0qcBJtr0Bq0aaKRCaj4Dur6m3k4iIB6nI97ft0y+IVIZlmfFqHn7YBJuhQ2HdugoGm62zYeUIE2ziR0PCbAUbEREvpnAjXsuy4KGH4KWXzOuZM+HNN6FWrQqcZNO/YdUowIIWY6D7THDo10JExJvpX3HxWk88Ac8+a9ZnzjSD9FVowN8/psGPY8x6q/uh60sKNiIiPkD/kotXeu4507UbzDxRt99ewRP8/iysu9+st30IznteUyGIiPgIhRvxOjNmwAMPmPWnnjJtbirk1ydh/YNmvd0k6Pi0go2IiA9RP1fxKnPmwN13m/WHH4ZSphU7k2XBhknw6xPmdYcnod3DLi+jiIjYS+FGvMa778LJmTe4/37T5qbcLAvWPwQbTzbS6fwctBnv8jKKiIj9FG7EK3z4Idx8s8kod9wBz1ekiYxlwbpxkDTNvO7yIrS6x11FFRERmynciMf75BMYPNiMYzN8uGlzU/5g44Qfx8Lmf5vX3V6BFne4rawiImI/hRvxaImJcN11cOKEmeF79mwzw3e5WE5YfTtsfQ1wQMJrEH+LO4srIiIeQOFGPNby5XDNNZCdDddeawboK/f8qM48WHUrbH/djF1z/jxodrM7iysiIh5CXcGlauRmQOY+0/6lHNasgb59ITMTrrgCFiwwk2GWizPXTIC5/XVw+EOPtxVsRESqEdXciPvlHocl7SBjBwTUgFotoXar0362hEAzEdovv0CfPpCeDhdfDIsWQXBwOT/LeQKW3wi73wdHAFwwH5oMdNuliYiI51G4Effb+a4JNmBqcI78ZJbThUSTGdCK9V+0YmSPljhqt+KxqS0JC2kGlKPaJi8bvr8B9n4MfkHwl/ehcT9XXomIiHgBh2WV8zmBj6jIlOniApYFn3aGoz9Dx6cgdiCkJUH6JvMzfz0rpeRzOAKgVnwxNT6tICTSdJ3Ky4Jvr4P9n4JfMFy0GBpeWWWXKSIi7lWR72/V3Ih7HfzeBBv/UDjnDgiuZ4LJaXZvO8qYmzdT25HEXzokMXLgJoKzTwafvOOFQWjvf4seGFjbhBxnNhz9xXzOxf+F6F5VdIEiIuJpFG7EvZKmm59xN5lgU4x9++CyPnXYsqUbLVp0418TITj65JuWEzL3nlnTk5ZkHnWdSIM/15h9A2rAxUsg6mK3X5aIiHguhRtxn4zdsGeRWW81tthdDh6E3r1hyxaIizPj2kRHn7KDww9qxJrl9NqYvCxI32rCTuZuiL4Cwlu75VJERMR7KNyI+2x+Baw8iLwE6rQ/4+0jR+Dyy2HjRmjUCJYtg9jYCpzfPwTqnGsWERGRkzTOjbhH7nHY+h+zXsw8TmlpcOWV8PPPEBVlgk2zZlVcRhER8UkKN+IeO+dD9mEIawKNinbHzsiAv/4VVq+G+vXhyy+hZUubyikiIj5H4UZcz7Jg08mGxC3HgF/h08+sLOjfH777DsLD4YsvoF07e4opIiK+SeFGXO/gcjiy3nTLjh9VsDknB66/3tTU1KgBn34K551nXzFFRMQ3KdyI6+XX2sQNLej+nZsLQ4fCkiUQEgL/+x/06GFjGUVExGcp3IhrZe6B3R+a9Zam+7dlwahR8P77EBQEixfDJZfYVkIREfFxCjfiWgXdvy+Guh0AeO01eP118PeH994zk2KKiIi4i8KNuE5eFmwp2v07KQnuu89smjIFrr3WnqKJiEj1oXAjrrNzPmQfgrBYaHQNOTmmnU1mJlx2Gfz973YXUEREqgOFG3ENyyqcR+pk9++JE2HtWqhbF954A/z0t01ERKqAvm7ENQ79AEd+MlMixI/iq6/g2WfNW6+9ZqZXEBERqQoKN+IaSYXdv//MqM/NNxf2krruOnuLJiIi1YvCjVRe5h7Y/QEAVoux3H477N0LLVrACy/YXDYREal2FG6k8jbPPNn9+yLmfdSR99+HgAB45x2oWdPuwomISHWjcCOVc0r37/2172GsGbePJ56Arl1tLJeIiFRbCjdSOTsXQPZBrNBYrht7LRkZcPHF8I9/2F0wERGprhRu5Oyd0v176Y67WLk6gDp14M03zWjEIiIidlC4kbN3aAUcWYeTYIZOMrN//+c/EBtrc7lERKRaU7iRs3ey1mbhmqEcSo9gxAj429/sLZKIiIjCjZydzL1YJ7t/T1k0lvh4mD7d5jKJiIgAAXYXQLzU5pk4rFy+/eNCft3TieXLoVYtuwslIiKimhs5G3nZ5CW9CsD0z+9h8mRISLC5TCIiIicp3EiF5W1fgH/uQXYfbsyh4P489JDdJRIRESmkcCMVY1kkf2sa18z5/i5efzNA3b5FRMSjKNxIhfzy1UoahawlKyeY9v1H07Sp3SUSEREpSuFGyi01FXZ8bmptVh+4keuGRNhcIhERkTMp3Ei5PTxuH33PfR+A84aMtbk0IiIixVO4kXJ5+22ITJ1JYEAuacF/oWZsZ7uLJCIiUiyPCDczZswgLi6OkJAQEhISWL16dYn7XnLJJTgcjjOWq6++ugpLXL3s2AH33ZPN7b1M9+/aXVVrIyIinsv2cLNgwQLGjRvHpEmTWLduHR07dqRPnz4cOHCg2P0//PBD9u/fX7D8+uuv+Pv78zeN++8Wublw001wZdv3iAo/gBXaCGIH2F0sERGREtkebqZOncro0aMZOXIkbdu2ZebMmYSFhTFnzpxi969Xrx7R0dEFy9KlSwkLC1O4cZMpU2D5cov7+5qGxI6Wd4FfoM2lEhERKZmt4SYnJ4e1a9fSu3fvgm1+fn707t2bFStWlOscs2fPZvDgwdSoUaPY97Ozs0lLSyuySPmsXIkZfficVZwX9yP4BUP8aLuLJSIiUipbw82hQ4fIy8sjKiqqyPaoqCiSk5PLPH716tX8+uuvjBo1qsR9pkyZQnh4eMESGxtb6XJXB2lpMHQo5OXB1NtOzogZNwRCGthbMBERkTLY/liqMmbPnk379u3p3r17iftMmDCB1NTUgmX37t1VWELvdc89sG0bdGu3jx6NF5qNLdWQWEREPJ+ts4JHRETg7+9PSkpKke0pKSlER0eXemxGRgbz58/n8ccfL3W/4OBggoODK13W6mTBAnj9dfDzg/envIojLRcaXAD1zrO7aCIiImWyteYmKCiILl26kJiYWLDN6XSSmJhIjx49Sj124cKFZGdnc9NNN7m7mNXKrl1w++1mfeIj2TTJMd2/VWsjIiLewvbHUuPGjWPWrFm8/vrrbNy4kTvvvJOMjAxGjhwJwLBhw5gwYcIZx82ePZv+/ftTv379qi6yz8rLg5tvNtMsJCTAwzcvhKwUCG0IsdfZXTwREZFysfWxFMCgQYM4ePAgEydOJDk5mU6dOvHZZ58VNDLetWsXfn5FM1hSUhLff/89X3zxhR1F9lnPPgvffgs1a5oRiQO2vGTeaHGnun+LiIjXcFiWZVXkgLlz51KzZs0zxpVZuHAhmZmZDB8+3KUFdLW0tDTCw8NJTU2ldu3adhfHY6xZAz17mkH75s2D4Vevgi/OB78g6L8bQiLtLqKIiFRjFfn+rvBjqSlTphARceZs0JGRkTz99NMVPZ14iLFjTbC54QYYNgzYdLLWpukQBRsREfEqFQ43u3btolmzZmdsb9q0Kbt27XJJoaRq/fQTrFoFgYEwfTo4svbDrvfMm63UkFhERLxLhcNNZGQkv/zyyxnbf/75ZzXu9VKvnuwQdd11EBUFbH4VnCcgoifU62Jr2URERCqqwuFmyJAh3HPPPXz11Vfk5eWRl5fHsmXLuPfeexk8eLA7yihulJ5uGg/DyS7geTmwZabZoO7fIiLihSrcW+qJJ55gx44d9OrVi4AAc7jT6WTYsGFqc+OF3n0Xjh2Dli3hkkuAHad0/24y0O7iiYiIVFiFw01QUBALFizgySefZP369YSGhtK+fXuaNm3qjvKJG1kWzDxZSXP77eBwUNiQ+Jw71P1bRES80lmPc9OiRQtatGjhyrJIFfvxR9OYODgYhg8HDq2Gw6tM9+9zbrO7eCIiImelwuFm4MCBdO/enQcffLDI9meffZY1a9awcOFClxVOTpHyDSwfZIJHSDSExkBo9Mn1aAg57bV/SJmnzG9I/Le/Qf36wA/53b8HQ2hUiceJiIh4sgqHm2+//ZbHHnvsjO19+/bl+eefd0WZpDi/TzFtYQAyyzGzeWCdU8JOTGHoObmenhvN0v9G43DU5/bb/eB4MuxaYI5VQ2IREfFiFQ43x44dIygo6IztgYGBpKWluaRQcprMfZC81Kxf9BFgmTByfD9kJZvl1NfOHDhx1CxpfxR7ylrAzhfgRF4AAQej4IuAk92/e0D9rlV0YSIiIq5X4XDTvn17FixYwMSJE4tsnz9/Pm3btnVZweQUO94CywkNLoDG15S+r2WZUHN8vwk8+cEnq/C1dXw/R/cnUzfsMIH+uXB8b+Hxre9366WIiIi4W4XDzaOPPsp1113H1q1bueyyywBITEzknXfe4f3333d5Aas9y4Jt88x6s3LM2+VwQFBds4QXHzZX/AAX/BVq18xh16YDhAedDD5+QRBzhevKLiIiYoMKh5t+/fqxePFinn76ad5//31CQ0Pp2LEjy5Yto169eu4oY/X254+QttE0EG5yg0tOWTAi8fVBhMc0Bhq75LwiIiKe4Ky6gl999dVcffXVgJml891332X8+PGsXbuWvLw8lxaw2suvtWk8AILCK326P/+E905OG3X77ZU+nYiIiMep8PQL+b799luGDx9Ow4YNef7557nssstYuXKlK8smedmw812zXp5HUuXwxhuQlQUdO0JCgktOKSIi4lEqVHOTnJzMvHnzmD17Nmlpadxwww1kZ2ezePFiNSZ2h73/g5wjZiqE6N6VPp1lFT6SKhiRWERExMeUu+amX79+tGrVil9++YVp06axb98+XnrpJXeWTQoaEt8Mfv6VPt1338Eff0CNGjB0aKVPJyIi4pHKXXPz6aefcs8993DnnXdq2oWqcDwF9n9q1l30SCp/Hqkbb4TatV1yShEREY9T7pqb77//nvT0dLp06UJCQgIvv/wyhw4dcmfZqred74CVB/W7Q3ibSp/u0CH44AOzrobEIiLiy8odbs4//3xmzZrF/v37uf3225k/fz4NGzbE6XSydOlS0tPT3VnO6qciY9uUw7x5kJMDXbqYRURExFdVuLdUjRo1uOWWW/j+++/ZsGEDf//733nmmWeIjIzkmmvKGD1XyufIejj6ixlUr+ngSp/O6SxsSHzHHZU+nYiIiEc7667gAK1ateLZZ59lz549vPvuu64qk2x73fxsdA0EV35gxK++gi1boFYtGFz5rCQiIuLRKhVu8vn7+9O/f38+/vhjV5yuenOegB1vm/XmrnkklV9rc9NNULOmS04pIiLisVwSbsSF9n0K2QchJApi+lT6dCkpsGiRWVdDYhERqQ4UbjzN9pOPpOKGgl9gpU83Zw7k5sL555tRiUVERHydwo0nyT4Me/9r1l3QS8rphFmzzLpqbUREpLpQuPEkO941bW7qdoa6HSp9uqVLYft2qFMHbnDNhOIiIiIeT+HGk2yfZ366eETiYcMgLMwlpxQREfF4Cjee4uhv8OdacARA3I2VPt2+ffDfk0+49EhKRESqE4UbT5HfkLjR1RDSoNKnmz0b8vLgwgtBE7aLiEh1onDjCZy5sP1Ns+6CR1J5eWpILCIi1ZfCjSdIXgpZyRBcHxpeXenTffop7N4N9evDwIEuKJ+IiIgXUbjxBPnTLTS9EfyDKn26/BGJhw+HkJBKn05ERMSrKNzYLecI7Fls1puPqPTpdu2CTz4x67fdVunTiYiIeB2FG7vtfA+c2RDezoxvU0mvvWYG77v0UmjVygXlExER8TIKN3bL7yXVfDg4HJU61YkTJtyAGhKLiEj1pXBjp7QkOLQCHP5mLqlK+t//YP9+aNAABgxwQflERES8kMKNnba/YX7G9IHQmEqfLr8h8S23QFDl2yWLiIh4JYUbuzjzCsONC8a22bYNvvjCrI8eXenTiYiIeC2FG7sc+Aoy90BgHWh8TaVPN2sWWBZccQXEx1e+eCIiIt5K4cYuBWPbDAb/yg1Gk5MDc+aYdTUkFhGR6k7hxg4n0mD3B2a9eeUfSX30ERw4ANHR0K9fpU8nIiLi1RRu7LDrfcg7DrVbQf2ESp9u5kzzc9QoCAys9OlERES8msKNHfLHtmlW+bFtNm+GZcvMaUaNckHZREREvJzCTVU7tg0OfAs4oNnNlT7df/5jfvbtC02bVvp0IiIiXk/hpqptO9n9O7o3hDWu1KmysmDuXLN+xx2VLJeIiIiPULipSpbTpWPbfPghHD4MjRubmhsRERHxgHAzY8YM4uLiCAkJISEhgdWrV5e6/9GjRxkzZgwxMTEEBwfTsmVLPsmfBtvTHfgOMrZDQC2Irfz8CPkjEo8aBQEBlT6diIiIT7D1K3HBggWMGzeOmTNnkpCQwLRp0+jTpw9JSUlERkaesX9OTg6XX345kZGRvP/++zRq1IidO3dSp06dqi/82chvSNz0BggIq9Spfv8dvv0W/Pzg1ltdUDYREREfYWu4mTp1KqNHj2bkyJEAzJw5kyVLljBnzhweeuihM/afM2cOf/75Jz/88AOBJ/s8x8XFlfoZ2dnZZGdnF7xOS0tz3QVURG4G7Fpo1l3wSCq/IXG/fuaxlIiIiBi2PZbKyclh7dq19O7du7Awfn707t2bFStWFHvMxx9/TI8ePRgzZgxRUVG0a9eOp59+mry8vBI/Z8qUKYSHhxcssbGxLr+Wctn9IeQeg5rNocFfKnWq48fh9ZOVQBqRWEREpCjbws2hQ4fIy8sjKiqqyPaoqCiSk5OLPWbbtm28//775OXl8cknn/Doo4/y/PPP8+STT5b4ORMmTCA1NbVg2b17t0uvo9y2uW5sm4UL4ehR0/X7iisqXzQRERFf4lXNUJ1OJ5GRkfznP//B39+fLl26sHfvXp577jkmTZpU7DHBwcEEBwdXcUlPk7ELUpaZ9WbDKn26/BGJb7sN/P0rfToRERGfYlu4iYiIwN/fn5SUlCLbU1JSiI6OLvaYmJgYAgMD8T/lG71NmzYkJyeTk5NDUFCQW8t81ra/CVgQeQnUjKvUqTZsgBUrTO+oW25xReFERER8i22PpYKCgujSpQuJiYkF25xOJ4mJifTo0aPYYy644AK2bNmC0+ks2LZp0yZiYmI8N9hYVmEvKRdMkpnf/fvaa81EmSIiIlKUrePcjBs3jlmzZvH666+zceNG7rzzTjIyMgp6Tw0bNowJEyYU7H/nnXfy559/cu+997Jp0yaWLFnC008/zZgxY+y6hLIdWgHpmyGgBsReX6lTZWTAm2+adY1ILCIiUjxb29wMGjSIgwcPMnHiRJKTk+nUqROfffZZQSPjXbt24edXmL9iY2P5/PPPuf/+++nQoQONGjXi3nvv5cEHH7TrEsqWX2sTOxACa1bqVPPnQ1oaxMfDZZe5oGwiIiI+yGFZlmV3IapSWloa4eHhpKamUrt2bfd+WO5xWBQDJ1LhskSIrlwi6d4d1qyBf/4THnjARWUUERHxAhX5/rZ9+gWftucjE2zCmkDUJZU61bp1JtgEBsLJp3YiIiJSDIUbd8p/JNVsGDgq90f9wQfm54AB0KBBJcslIiLiwxRu3CVzHyR/YdZd0EtqzRrzU21tRERESqdw4y473gLLCQ0ugFrnVOpUlgU//mjWu3VzQdlERER8mMKNO5w6to0LJsncvh2OHIGgIGjXrtKnExER8WkKN+7w54+Q+jv4h0CTGyp9uvxam44dTcARERGRkincuEP+JJmNB0BQeKVPl9/epmvXSp9KRETE5yncuFpeNux816y74JEUFNbcKNyIiIiUTeHG1fb+D3L+hNCGEN270qdzOmHtWrOucCMiIlI2hRtXK2hIfDP4+Ze+bzls3gzp6RAaCm3bVvp0IiIiPk/hxpWOp8C+T8y6ix9Jde4MAbbOBCYiIuIdFG5caec7YOVB/e4Q3sYlp1R7GxERkYpRuHGlba4b2yafekqJiIhUjMKNqxxZD0d/Br8gaDrYJafMzYWffjLrCjciIiLlo1YcrpKbCRE9IKwxBNdzySn/+AMyM6FmTWjZ0iWnFBER8XkKN67SoCdc8QPk5bjslPntbc47D/wr3/FKRESkWtBjKVfzd938CJosU0REpOIUbjyYekqJiIhUnMKNh8rJgfXrzbrCjYiISPkp3Hio336D7GwID4f4eLtLIyIi4j0UbjzUqY+kHA57yyIiIuJNFG48lNrbiIiInB2FGw+lcCMiInJ2FG48UFYWbNhg1tUNXEREpGIUbjzQhg1w4gRERECTJnaXRkRExLso3HigUyfLVGNiERGRilG48UBqbyMiInL2FG48kMKNiIjI2VO48TCZmWYAP1C4ERERORsKNx5m/XpwOiEmBho1srs0IiIi3kfhxsPokZSIiEjlKNx4mFN7SomIiEjFKdx4GNXciIiIVI7CjQdJS4OkJLPepYu9ZREREfFWCjce5KefwLIgNhaiouwujYiIiHdSuPEgeiQlIiJSeQo3HiQ/3GiyTBERkbOncONB1FNKRESk8hRuPMSRI7B1q1lXY2IREZGzp3DjIdauNT+bN4d69ewti4iIiDdTuPEQakwsIiLiGgo3HkLhRkRExDUUbjyEekqJiIi4hsKNBzh4EHbuNOvnnWdvWURERLydwo0HyK+1adUKate2tywiIiLeTuHGA6i9jYiIiOt4RLiZMWMGcXFxhISEkJCQwOrVq0vcd968eTgcjiJLSEhIFZbW9RRuREREXMf2cLNgwQLGjRvHpEmTWLduHR07dqRPnz4cOHCgxGNq167N/v37C5ad+Q1WvJTCjYiIiOvYHm6mTp3K6NGjGTlyJG3btmXmzJmEhYUxZ86cEo9xOBxER0cXLFFePIX2vn1m8fODzp3tLo2IiIj3szXc5OTksHbtWnr37l2wzc/Pj969e7NixYoSjzt27BhNmzYlNjaWa6+9lt9++63EfbOzs0lLSyuyeJL8kYnbtoUaNewti4iIiC+wNdwcOnSIvLy8M2peoqKiSE5OLvaYVq1aMWfOHD766CPeeustnE4nPXv2ZM+ePcXuP2XKFMLDwwuW2NhYl19HZWiyTBEREdey/bFURfXo0YNhw4bRqVMnLr74Yj788EMaNGjAq6++Wuz+EyZMIDU1tWDZvXt3FZe4dGpvIyIi4loBdn54REQE/v7+pKSkFNmekpJCdHR0uc4RGBhI586d2bJlS7HvBwcHExwcXOmyuoNlKdyIiIi4mq01N0FBQXTp0oXExMSCbU6nk8TERHr06FGuc+Tl5bFhwwZiYmLcVUy32b3bjE4cEAAdOthdGhEREd9ga80NwLhx4xg+fDhdu3ale/fuTJs2jYyMDEaOHAnAsGHDaNSoEVOmTAHg8ccf5/zzz+ecc87h6NGjPPfcc+zcuZNRo0bZeRlnJb/Wpl07CA21tywiIiK+wvZwM2jQIA4ePMjEiRNJTk6mU6dOfPbZZwWNjHft2oWfX2EF05EjRxg9ejTJycnUrVuXLl268MMPP9C2bVu7LuGsabJMERER13NYlmXZXYiqlJaWRnh4OKmpqdS2eSKnyy+HL7+EV1+F226ztSgiIiIerSLf317XW8pXqDGxiIiIeyjc2GTbNjh6FIKCTJsbERERcQ2FG5vk19p07GgCjoiIiLiGwo1N9EhKRETEPRRubKKeUiIiIu6hcGMDp7NwwkzV3IiIiLiWwo0NNm2C9HQzcF+bNnaXRkRExLco3Ngg/5FU585m6gURERFxHYUbG6gxsYiIiPso3NhA4UZERMR9FG6qWG4u/PSTWVe4ERERcT2Fmyr2xx+QmQk1a0KrVnaXRkRExPco3FSxNWvMzy5dwE9/+iIiIi6nr9cqpvY2IiIi7qVwU8UUbkRERNxL4aYK5eTAzz+bdYUbERER91C4qUK//QbZ2VCnDsTH210aERER36RwU4VOfSTlcNhbFhEREV+lcFOF8ntK6ZGUiIiI+yjcVCE1JhYREXE/hZsqkpUFGzaYdYUbERER91G4qSK//GKmXoiIgCZN7C6NiIiI71K4qSJqTCwiIlI1FG6qiNrbiIiIVA2FmyqSH266dbO3HCIiIr5O4aYKZGSYAfxANTciIiLupnBTBdavB6cTYmKgYUO7SyMiIuLbFG6qgNrbiIiIVB2FmyqgcCMiIlJ1FG6qgMKNiIhI1VG4cbO0NEhKMusKNyIiIu6ncONm69aBZZlRiSMj7S6NiIiI71O4cTM9khIREalaCjdupnAjIiJStRRu3EzhRkREpGop3LjRkSOwdatZ79LF3rKIiIhUFwo3brR2rfkZHw/16tlbFhERkepC4caN1qwxP/VISkREpOoo3LiR2tuIiIhUPYUbN1K4ERERqXoKN25y4ADs2mXWzzvP3rKIiIhUJwo3bpLfmLhVK6hd296yiIiIVCcKN26iR1IiIiL2ULhxk/yeUt262VsOERGR6kbhxk1UcyMiImIPhRs32LcP9u8HPz/o1Mnu0oiIiFQvCjdukF9r07Yt1Khhb1lERESqG48INzNmzCAuLo6QkBASEhJYvXp1uY6bP38+DoeD/v37u7eAFaRHUiIiIvaxPdwsWLCAcePGMWnSJNatW0fHjh3p06cPBw4cKPW4HTt2MH78eC688MIqKmn5KdyIiIjYx/ZwM3XqVEaPHs3IkSNp27YtM2fOJCwsjDlz5pR4TF5eHkOHDmXy5Mk0b968CktbNssqDDfqKSUiIlL1bA03OTk5rF27lt69exds8/Pzo3fv3qxYsaLE4x5//HEiIyO59dZby/yM7Oxs0tLSiizutGsXHDwIAQHQoYNbP0pERESKYWu4OXToEHl5eURFRRXZHhUVRXJycrHHfP/998yePZtZs2aV6zOmTJlCeHh4wRIbG1vpcpcmv9amfXsICXHrR4mIiEgxbH8sVRHp6encfPPNzJo1i4iIiHIdM2HCBFJTUwuW3bt3u7WMam8jIiJirwA7PzwiIgJ/f39SUlKKbE9JSSE6OvqM/bdu3cqOHTvo169fwTan0wlAQEAASUlJxMfHFzkmODiY4OBgN5S+eAo3IiIi9rK15iYoKIguXbqQmJhYsM3pdJKYmEiPHj3O2L9169Zs2LCB9evXFyzXXHMNl156KevXr3f7I6eynNqYWOFGRETEHrbW3ACMGzeO4cOH07VrV7p37860adPIyMhg5MiRAAwbNoxGjRoxZcoUQkJCaNeuXZHj69SpA3DGdjts2wZHj0JQEHhAcURERKol28PNoEGDOHjwIBMnTiQ5OZlOnTrx2WefFTQy3rVrF35+3tE0KH+yzE6dTMARERGRquewLMuyuxBVKS0tjfDwcFJTU6ldu7ZLzz1+PDz/PNx1F8yY4dJTi4iIVGsV+f72jioRL6H2NiIiIvZTuHERpxPWrjXrCjciIiL2UbhxkU2b4NgxCA2FNm3sLo2IiEj1ZXuDYl+xdy/Urw+tWpmpF0RERMQe+hp2kV69zJxSbp66SkRERMqgx1Iu5HBAeLjdpRAREaneFG5ERETEpyjciIiIiE9RuBERERGfonAjIiIiPkXhRkRERHyKwo2IiIj4FIUbERER8SkKNyIiIuJTFG5ERETEpyjciIiIiE9RuBERERGfonAjIiIiPkXhRkRERHxKgN0FqGqWZQGQlpZmc0lERESkvPK/t/O/x0tT7cJNeno6ALGxsTaXRERERCoqPT2d8PDwUvdxWOWJQD7E6XSyb98+atWqhcPhsLs4bpOWlkZsbCy7d++mdu3adhfH7arT9epafVd1ul5dq+9y1/ValkV6ejoNGzbEz6/0VjXVrubGz8+Pxo0b212MKlO7du1q8cuUrzpdr67Vd1Wn69W1+i53XG9ZNTb51KBYREREfIrCjYiIiPgUhRsfFRwczKRJkwgODra7KFWiOl2vrtV3Vafr1bX6Lk+43mrXoFhERER8m2puRERExKco3IiIiIhPUbgRERERn6JwIyIiIj5F4cYLTZkyhW7dulGrVi0iIyPp378/SUlJpR4zb948HA5HkSUkJKSKSlw5jz322Bllb926danHLFy4kNatWxMSEkL79u355JNPqqi0lRMXF3fGtTocDsaMGVPs/t52X7/99lv69etHw4YNcTgcLF68uMj7lmUxceJEYmJiCA0NpXfv3mzevLnM886YMYO4uDhCQkJISEhg9erVbrqC8ivtWk+cOMGDDz5I+/btqVGjBg0bNmTYsGHs27ev1HOeze9CVSjrvo4YMeKMcl955ZVlntcT7yuUfb3F/Q47HA6ee+65Es/pife2PN81WVlZjBkzhvr161OzZk0GDhxISkpKqec929/zilC48ULffPMNY8aMYeXKlSxdupQTJ05wxRVXkJGRUepxtWvXZv/+/QXLzp07q6jElXfuuecWKfv3339f4r4//PADQ4YM4dZbb+Wnn36if//+9O/fn19//bUKS3x21qxZU+Q6ly5dCsDf/va3Eo/xpvuakZFBx44dmTFjRrHvP/vss0yfPp2ZM2eyatUqatSoQZ8+fcjKyirxnAsWLGDcuHFMmjSJdevW0bFjR/r06cOBAwfcdRnlUtq1ZmZmsm7dOh599FHWrVvHhx9+SFJSEtdcc02Z563I70JVKeu+Alx55ZVFyv3uu++Wek5Pva9Q9vWeep379+9nzpw5OBwOBg4cWOp5Pe3elue75v777+e///0vCxcu5JtvvmHfvn1cd911pZ73bH7PK8wSr3fgwAELsL755psS95k7d64VHh5edYVyoUmTJlkdO3Ys9/433HCDdfXVVxfZlpCQYN1+++0uLpn73XvvvVZ8fLzldDqLfd+b7ytgLVq0qOC10+m0oqOjreeee65g29GjR63g4GDr3XffLfE83bt3t8aMGVPwOi8vz2rYsKE1ZcoUt5T7bJx+rcVZvXq1BVg7d+4scZ+K/i7YobhrHT58uHXttddW6DzecF8tq3z39tprr7Uuu+yyUvfxhnt7+nfN0aNHrcDAQGvhwoUF+2zcuNECrBUrVhR7jrP9Pa8o1dz4gNTUVADq1atX6n7Hjh2jadOmxMbGcu211/Lbb79VRfFcYvPmzTRs2JDmzZszdOhQdu3aVeK+K1asoHfv3kW29enThxUrVri7mC6Vk5PDW2+9xS233FLqJK/efF9PtX37dpKTk4vcu/DwcBISEkq8dzk5Oaxdu7bIMX5+fvTu3dvr7ndqaioOh4M6deqUul9Ffhc8yddff01kZCStWrXizjvv5PDhwyXu60v3NSUlhSVLlnDrrbeWua+n39vTv2vWrl3LiRMnityn1q1b06RJkxLv09n8np8NhRsv53Q6ue+++7jgggto165difu1atWKOXPm8NFHH/HWW2/hdDrp2bMne/bsqcLSnp2EhATmzZvHZ599xiuvvML27du58MILSU9PL3b/5ORkoqKiimyLiooiOTm5KorrMosXL+bo0aOMGDGixH28+b6eLv/+VOTeHTp0iLy8PK+/31lZWTz44IMMGTKk1IkGK/q74CmuvPJK3njjDRITE/nnP//JN998Q9++fcnLyyt2f1+5rwCvv/46tWrVKvNRjaff2+K+a5KTkwkKCjojkJd2n87m9/xsVLtZwX3NmDFj+PXXX8t8NtujRw969OhR8Lpnz560adOGV199lSeeeMLdxayUvn37Fqx36NCBhIQEmjZtynvvvVeu/w15q9mzZ9O3b18aNmxY4j7efF/FOHHiBDfccAOWZfHKK6+Uuq+3/i4MHjy4YL19+/Z06NCB+Ph4vv76a3r16mVjydxvzpw5DB06tMyG/p5+b8v7XeMpVHPjxe6++27+97//8dVXX9G4ceMKHRsYGEjnzp3ZsmWLm0rnPnXq1KFly5Yllj06OvqM1vopKSlER0dXRfFcYufOnXz55ZeMGjWqQsd5833Nvz8VuXcRERH4+/t77f3ODzY7d+5k6dKlpdbaFKes3wVP1bx5cyIiIkost7ff13zfffcdSUlJFf49Bs+6tyV910RHR5OTk8PRo0eL7F/afTqb3/OzoXDjhSzL4u6772bRokUsW7aMZs2aVfgceXl5bNiwgZiYGDeU0L2OHTvG1q1bSyx7jx49SExMLLJt6dKlRWo4PN3cuXOJjIzk6quvrtBx3nxfmzVrRnR0dJF7l5aWxqpVq0q8d0FBQXTp0qXIMU6nk8TERI+/3/nBZvPmzXz55ZfUr1+/wuco63fBU+3Zs4fDhw+XWG5vvq+nmj17Nl26dKFjx44VPtYT7m1Z3zVdunQhMDCwyH1KSkpi165dJd6ns/k9P9vCi5e58847rfDwcOvrr7+29u/fX7BkZmYW7HPzzTdbDz30UMHryZMnW59//rm1detWa+3atdbgwYOtkJAQ67fffrPjEirk73//u/X1119b27dvt5YvX2717t3bioiIsA4cOGBZ1pnXunz5cisgIMD617/+ZW3cuNGaNGmSFRgYaG3YsMGuS6iQvLw8q0mTJtaDDz54xnvefl/T09Otn376yfrpp58swJo6dar1008/FfQQeuaZZ6w6depYH330kfXLL79Y1157rdWsWTPr+PHjBee47LLLrJdeeqng9fz5863g4GBr3rx51u+//27ddtttVp06dazk5OQqv75TlXatOTk51jXXXGM1btzYWr9+fZHf4+zs7IJznH6tZf0u2KW0a01PT7fGjx9vrVixwtq+fbv15ZdfWuedd57VokULKysrq+Ac3nJfLavsv8eWZVmpqalWWFiY9corrxR7Dm+4t+X5rrnjjjusJk2aWMuWLbN+/PFHq0ePHlaPHj2KnKdVq1bWhx9+WPC6PL/nlaVw44WAYpe5c+cW7HPxxRdbw4cPL3h93333WU2aNLGCgoKsqKgo66qrrrLWrVtX9YU/C4MGDbJiYmKsoKAgq1GjRtagQYOsLVu2FLx/+rValmW99957VsuWLa2goCDr3HPPtZYsWVLFpT57n3/+uQVYSUlJZ7zn7ff1q6++Kvbvbv41OZ1O69FHH7WioqKs4OBgq1evXmf8OTRt2tSaNGlSkW0vvfRSwZ9D9+7drZUrV1bRFZWstGvdvn17ib/HX331VcE5Tr/Wsn4X7FLatWZmZlpXXHGF1aBBAyswMNBq2rSpNXr06DNCirfcV8sq+++xZVnWq6++aoWGhlpHjx4t9hzecG/L811z/Phx66677rLq1q1rhYWFWQMGDLD2799/xnlOPaY8v+eV5Tj5wSIiIiI+QW1uRERExKco3IiIiIhPUbgRERERn6JwIyIiIj5F4UZERER8isKNiIiI+BSFGxEREfEpCjciIiLiUxRuRKTaczgcLF682O5iiIiLKNyIiK1GjBiBw+E4Y7nyyivtLpqIeKkAuwsgInLllVcyd+7cItuCg4NtKo2IeDvV3IiI7YKDg4mOji6y1K1bFzCPjF555RX69u1LaGgozZs35/333y9y/IYNG7jssssIDQ2lfv363HbbbRw7dqzIPnPmzOHcc88lODiYmJgY7r777iLvHzp0iAEDBhAWFkaLFi34+OOP3XvRIuI2Cjci4vEeffRRBg4cyM8//8zQoUMZPHgwGzduBCAjI4M+ffpQt25d1qxZw8KFC/nyyy+LhJdXXnmFMWPGcNttt7FhwwY+/vhjzjnnnCKfMXnyZG644QZ++eUXrrrqKoYOHcqff/5ZpdcpIi7i0jnGRUQqaPjw4Za/v79Vo0aNIstTTz1lWZZlAdYdd9xR5JiEhATrzjvvtCzLsv7zn/9YdevWtY4dO1bw/pIlSyw/Pz8rOTnZsizLatiwofXwww+XWAbAeuSRRwpeHzt2zAKsTz/91GXXKSJVR21uRMR2l156Ka+88kqRbfXq1StY79GjR5H3evTowfr16wHYuHEjHTt2pEaNGgXvX3DBBTidTpKSknA4HOzbt49evXqVWoYOHToUrNeoUYPatWtz4MCBs70kEbGRwo2I2K5GjRpnPCZyldDQ0HLtFxgYWOS1w+HA6XS6o0gi4mZqcyMiHm/lypVnvG7Tpg0Abdq04eeffyYjI6Pg/eXLl+Pn50erVq2oVasWcXFxJCYmVmmZRcQ+qrkREdtlZ2eTnJxcZFtAQAAREREALFy4kK5du/KXv/yFt99+m9WrVzN79mwAhg4dyqRJkxg+fDiPPfYYBw8eZOzYsdx8881ERUUB8Nhjj3HHHXcQGRlJ3759SU9PZ/ny5YwdO7ZqL1REqoTCjYjY7rPPPiMmJqbItlatWvHHH38ApifT/Pnzueuuu4iJieHdd9+lbdu2AISFhfH5559z77330q1bN8LCwhg4cCBTp04tONfw4cPJysrihRdeYPz48URERHD99ddX3QWKSJVyWJZl2V0IEZGSOBwOFi1aRP/+/e0uioh4CbW5EREREZ+icCMiIiI+RW1uRMSj6cm5iFSUam5ERETEpyjciIiIiE9RuBERERGfonAjIiIiPkXhRkRERHyKwo2IiIj4FIUbERER8SkKNyIiIuJT/h8p7cVV+ZOHPAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib.ticker import FuncFormatter\n",
    "epoch = [i+1 for i in range(num_epochs)]\n",
    "plt.plot(epoch, train_acc, color='blue', label='train_acc')\n",
    "plt.plot(epoch, test_acc, color='orange', label='test_acc')\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('Acc')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35c9e838",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "cnn",
   "language": "python",
   "name": "cnn"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.19"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
